from gxl_ai_utils.utils import utils_file
import sys
import os
# input_scp_path = "/home/work_nfs8/xlgeng/new_workspace/wenet_gxl_salmonn4ft_LLM/examples/aishell/ft_LLM/data_list/all.text"
input_scp_path = "haha/text"
input_list_path = sys.argv[1]
# word_list_path = "/home/work_nfs8/xlgeng/new_workspace/gxl_ai_utils/eggs/cats_and_dogs/ngram_task/data_handler/haha/word_95w.list"
word_list_path = "/home/work_nfs15/asr_data/ckpt/xlgeng/word_seg/word_95w.list"
# output_dir = './gxl_data_zoo/all_data_new'
output_dir = sys.argv[2]
utils_file.makedir_sil(output_dir)

# 开始提取文本
# 1. 》》output_dir/text.list
# 2. cp list_path 》》output_dir/text.list
utils_file.logging_print("开始 提取文本")
output_path = utils_file.join_path(output_dir, "001-text.list")
command_line = f"echo input_list_path and input_scp_path must be specified at least one"
if len(input_list_path) != 0:
    command_line = f"cp {input_list_path} {output_path}"
else:
    command_line = f"python draw_text_from_scp.py --scp_path {input_scp_path} --output_path {output_path}"
utils_file.do_execute_shell_command(command_line)
utils_file.logging_print('提取文本 完成')



utils_file.logging_print('开始 去除空格')
input_path = output_path
output_path = utils_file.join_path(output_dir, "002-text_nospace.list")
command_line = f"sed 's/ //g' {input_path} > {output_path}"
utils_file.do_execute_shell_command(command_line)
utils_file.logging_print('去除空格 完成')


utils_file.logging_print('开始 字母转大写')
input_path = output_path
output_path = utils_file.join_path(output_dir, "003-text_nospace_ABC.list")
command_line = f"cat {input_path} | tr [:lower:] [:upper:] > {output_path}"
utils_file.do_execute_shell_command(command_line)
utils_file.logging_print('字母转大写 完成')

utils_file.logging_print('开始 去特殊符号')
input_path = output_path
output_path = utils_file.join_path(output_dir, "004-text_nospace_ABC_nospecial.list")
command_line = 'wc -l ' + input_path
row_num_str = utils_file.do_execute_shell_command(command_line)
print(row_num_str)
row_num = int(row_num_str.split(' ')[0])
command_line = f"python cn_tn.py {input_path} {output_path} {row_num}"
utils_file.do_execute_shell_command(command_line)
utils_file.logging_print('去特殊符号 完成')

utils_file.logging_print('开始 去长句')
input_path = output_path
output_path = utils_file.join_path(output_dir, "005-text_nospace_ABC_nospecial_nolong.list")
command_line = [
    "awk",
    "'{if(length($0)<=1000) print $0}'",
    f"{input_path}",
    ">",
    f"{output_path}"
]
utils_file.do_execute_shell_command(command_line)
utils_file.logging_print('去长句 完成')


utils_file.logging_print('开始 分词')
input_path = output_path
output_path = utils_file.join_path(output_dir, "006-text_nospace_ABC_nospecial_nolong_wordseg.list")
# command_line = f"/home/work_nfs3/fyu/tools/wordseg/word_seg4 {word_list_path} {input_path} {output_path}"
command_line = f"/home/work_nfs15/asr_data/ckpt/xlgeng/word_seg/word_seg4 {word_list_path} {input_path} {output_path}"
utils_file.do_execute_shell_command(command_line)
utils_file.logging_print('分词 完成')


input_path = output_path
output_path = utils_file.join_path(output_dir,'final_data.list')
utils_file.copy_file(input_path, output_path)

utils_file.logging_print('删除中间结果以节省存储')
utils_file.remove_file(utils_file.join_path(output_dir, "001-text.list"))
utils_file.remove_file(utils_file.join_path(output_dir, "002-text_nospace.list"))
utils_file.remove_file(utils_file.join_path(output_dir, "003-text_nospace_ABC.list"))
utils_file.remove_file(utils_file.join_path(output_dir, "004-text_nospace_ABC_nospecial.list"))
utils_file.remove_file(utils_file.join_path(output_dir, "005-text_nospace_ABC_nospecial_nolong.list"))
utils_file.remove_file(utils_file.join_path(output_dir, "006-text_nospace_ABC_nospecial_nolong_wordseg.list"))
